#!/usr/bin/env python
#coding=utf8
"""
@描述：大文件求top n问题
	   将文件拆分，分别排序，后合并；按照key值路由
@日期：2015年02月01日 星期日
@作者：hongxing.fan
"""

import sys
import gzip
import time

# 全局的map结构，内容是文件handler
handlerMap = dict()
testPath = "/home/hongxing.fan/test/"
pt = 20
num = 20000
partitionValue = dict()

# 按照key%r划分文件
def partition(key,r):
	i = hash(key) % r
	if(i not in handlerMap):
		handlerMap[i] = open(testPath + str(i),"a") # 追加写入
	return handlerMap[i]

# 写入记录
def write(keys,handler):
	for key in keys:
		handler.write(key)
	handler.flush()

# 读取gz文件,目前还不是很完善，需要一次写入多个行
def readGZ(path):
	handler = gzip.open(path,mode="rb") # read binary二进制
	for line in handler:
		cols = line.split("\t")
		key = cols[3]
		partitionNum = partition(key,pt)
		if(partitionNum not in partitionValue):
			partitionValue[partitionNum] = list()
		partitionValue[partitionNum].append(line)
		if(len(partitionValue[partitionNum]) > num):
			write(partitionValue[partitionNum],partitionNum)
			del(partitionValue[partitionNum][:])
	handler.close()

# 分别对几个文件排序,取出前10名
def sort(i):
	kv = dict()
	handler = open(testPath + str(i),"r")
	for line in handler:
		line = line.strip()
		cols = line.split("\t")
		key = cols[3]
		if key in kv:
			kv[key] += 1
		else:
			kv[key] = 1
	top = 10
	# 用于返回top n
	resultKV = dict()
	for (k,v) in sorted(kv.items(),key=lambda x:x[1],reverse = True):
		#print("%d\t%s\t%d" %(i,k,v))
		resultKV[k] = v
		top -= 1
		if(top < 0):
			break
	return resultKV
	
# 入口函数
if __name__ == "__main__":
	print("划分gz文件:" + time.strftime("%Y-%m-%d %X", time.localtime()))
	readGZ("/home/hongxing.fan/files/autolog_20140601.txt.gz")
	print("划分完gz文件:" + time.strftime("%Y-%m-%d %X", time.localtime()))
	wholeKV = dict()
	for i in range(10):
		for (k,v) in sort(i).items():
			if k in wholeKV:
				print("Error,No keys should in wholeKV: %s" %(k))
			else:
				wholeKV[k] = v
	top = 10
	for (k,v) in sorted(wholeKV.items(),key = lambda x:x[1],reverse=True): 
		print("%s\t%d" %(k,v))
		top -= 1
		if(top < 0):
			break
	print("程序结束:" + time.strftime("%Y-%m-%d %X", time.localtime()))
